# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import re
from utils.HttpUtils import HttpUtils

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
携程-酒店-民宿
https://inn.ctrip.com/onlineinn/list/shanghai2/
"""
class WineShop():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            "authority": "inn.ctrip.com",
            "method": "GET",
            "path": "/onlineinn/list/shanghai2/",
            "scheme": "https",
            "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "cookie": "_abtest_userid=809b2dc1-cc19-453d-b51a-e9118c3910da; _bfs=1.1; _RF1=119.114.17.37; _RSG=L74dPCFd6S9SnAQmkSOA7A; _RDG=28f8462926ed592bf031b6817ca98f627b; _RGUID=97069cf3-199c-41d5-a1ca-61cbb068ac5c; _ga=GA1.2.1707957158.1576639089; _gid=GA1.2.1369912887.1576639089; _bfi=p1%3D100101991%26p2%3D0%26v1%3D1%26v2%3D0; MKT_Pagesource=PC; _jzqco=%7C%7C%7C%7C%7C1.669710358.1576639093692.1576639093692.1576639093694.1576639093692.1576639093694.0.0.0.1.1; MKT_CKID=1576639093703.1dc7u.ekp2; MKT_CKID_LMT=1576639093705; __zpspc=9.1.1576639093.1576639093.1%232%7Csp0.baidu.com%7C%7C%7C%25E6%2590%25BA%25E7%25A8%258B%25E7%25BD%2591%7C%23; Session=SmartLinkCode=U155952&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=&SmartLinkLanguage=zh; GUID=09031121210974067355; nfes_isSupportWebP=1; nfes_isSupportWebP=1; Union=OUID=&AllianceID=66672&SID=1693366&SourceID=&AppID=&OpenID=&createtime=1576639126&Expires=1577243926440; _pd=%7B%22r%22%3A2%2C%22d%22%3A121%2C%22_d%22%3A119%2C%22p%22%3A157%2C%22_p%22%3A36%2C%22o%22%3A162%2C%22_o%22%3A5%2C%22s%22%3A164%2C%22_s%22%3A2%7D; _bfa=1.1576639086057.3rshkg.1.1576639086057.1576639299884.1.8.10650016818",
            "dnt": "1",
            "if-none-match": "1097b0-JfIGgaz0x2HUssBJs7HucSS8igA",
            "sec-fetch-mode": "navigate",
            "sec-fetch-site": "none",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
        }


    def get_contents(self, pageNum):
        postData = {
            "d1": "2020-01-31",
            "d2": "2020-02-29",
            "p": pageNum,
        }
        url = "https://inn.ctrip.com/onlineinn/list/shanghai2/"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        selector = etree.HTML(html.text)
        xpath = selector.xpath('//div[@class="list-item "]')
        for x in xpath:
            dict_data = dict()
            dict_data['url'] = "https://inn.ctrip.com" + x.xpath('./div[@class="list-item-con"]/a/@href')[0]
            pid = re.findall(r"detail/(.*?)\?", dict_data["url"])[0]
            dict_data['title'] = x.xpath('./div[@class="list-item-con"]/a/text()')[0]  # 标题
            dict_data['point_count'] = "".join(x.xpath('./div[@class="list-item-con"]/div[2]/span[1]/span//text()'))  # 评分
            dict_data['comment_count'] = "".join(x.xpath('./div[@class="list-item-con"]/div[2]/span[2]/span//text()'))  # 点评数量
            dict_data['tmp1'] = "".join(x.xpath('./div[@class="list-item-con"]/div[2]/span[3]//text()'))  # 几室
            dict_data['tmp2'] = "".join(x.xpath('./div[@class="list-item-con"]/div[2]/span[4]//text()'))  # 几人
            dict_data['tmp3'] = "".join(x.xpath('./div[@class="list-item-con"]/div[2]/span[5]//text()'))  # 几床
            dict_data['address'] = "".join(x.xpath('./div[@class="list-item-con"]/div[2]/span[6]//text()'))  # 地址
            dict_data['price_now'] = "".join(x.xpath('./div[@class="list-item-con"]/div[3]/span[1]//text()'))  # 价格
            dict_data['price_origin'] = "".join(x.xpath('./div[@class="list-item-con"]/div[3]/span[2]//text()'))  # 原价
            # 获取评论内容
            self.get_comments(pid, 0)
            # 房屋详情
            # self.get_contents_1(url)

    # 房屋详情
    def get_contents_1(self, url):
        postData = {

        }
        url = "https://inn.ctrip.com/onlineinn/detail/861848781?d1=2019-12-18&d2=2019-12-19&nav=shanghai2"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        selector = etree.HTML(html.text)
        # 基础设施
        xpath = selector.xpath('//div[@class="con-list is-fold"]/div[2]')
        dict_data = dict()
        dict_data['type_1'] = "基础设施"
        dict_data['type_name_1'] = ""
        for x in selector.xpath('//div[@class="con-list is-fold"][1]/div[2]'):
            dict_data['type_name_1'] += "|".join(x.xpath('./div//text()')) + "|"
        dict_data['type_2'] = "卫浴设施"
        dict_data['type_name_2'] = ""
        for x in selector.xpath('//div[@class="con-list is-fold"][2]/div[2]'):
            dict_data['type_name_2'] += "|".join(x.xpath('./div//text()')) + "|"
        dict_data['type_2'] = "厨房设施"
        dict_data['type_name_2'] = ""
        for x in selector.xpath('//div[@class="con-list is-fold"][3]/div[2]'):
            dict_data['type_name_2'] += "|".join(x.xpath('./div//text()')) + "|"

    # 评论内容
    def get_comments(self, pid, pageNum):
        headers = {
            "authority": "m.ctrip.com",
            "method": "POST",
            "path": "/restapi/soa2/12455/json/BnbCommentList?_fxpcqlniredt=09031121210974067355",
            "scheme": "https",
            "accept": "*/*",
            "accept-encoding": "gzip, deflate, br",
            "accept-language": "zh-CN,zh;q=0.9",
            "content-type": "application/json",
            "cookie": "_abtest_userid=809b2dc1-cc19-453d-b51a-e9118c3910da; _RF1=119.114.17.37; _RSG=L74dPCFd6S9SnAQmkSOA7A; _RDG=28f8462926ed592bf031b6817ca98f627b; _RGUID=97069cf3-199c-41d5-a1ca-61cbb068ac5c; _ga=GA1.2.1707957158.1576639089; _gid=GA1.2.1369912887.1576639089; _bfi=p1%3D100101991%26p2%3D0%26v1%3D1%26v2%3D0; MKT_Pagesource=PC; _jzqco=%7C%7C%7C%7C%7C1.669710358.1576639093692.1576639093692.1576639093694.1576639093692.1576639093694.0.0.0.1.1; MKT_CKID=1576639093703.1dc7u.ekp2; MKT_CKID_LMT=1576639093705; __zpspc=9.1.1576639093.1576639093.1%232%7Csp0.baidu.com%7C%7C%7C%25E6%2590%25BA%25E7%25A8%258B%25E7%25BD%2591%7C%23; Session=SmartLinkCode=U155952&SmartLinkKeyWord=&SmartLinkQuary=&SmartLinkHost=&SmartLinkLanguage=zh; nfes_isSupportWebP=1; Union=OUID=&AllianceID=66672&SID=1693366&SourceID=&AppID=&OpenID=&createtime=1576639126&Expires=1577243926440; _gat=1; _bfa=1.1576639086057.3rshkg.1.1576639086057.1576674777986.3.20.10650016818",
            "cookieorigin": "https://inn.ctrip.com",
            "dnt": "1",
            "origin": "https://inn.ctrip.com",
            "referer": "https://inn.ctrip.com/onlineinn/detail/861848781?d1=2019-12-18&d2=2019-12-19&nav=shanghai2",
            "sec-fetch-mode": "cors",
            "sec-fetch-site": "same-site",
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36"
        }
        postData = {
            "pid":pid,
            "searchType":2,
            "tagId":0,
            "pageindex":pageNum, # 0
            "pageSize":10,  # 默认5
            "order":0,
            "head":{"cid":"09031121210974067355","ctok":"","cver":"1.0","lang":"01","sid":"8888","syscode":"09","auth":"","xsid":"","extension":[{"name":"webp","value":"1"},{"name":"cityid","value":""},{"name":"platform","value":"pc"},{"name":"source","value":"2"},{"name":"pagecode","value":"10650016818"},{"name":"module","value":"11"},{"name":"terminaltype","value":"100"}]}
        }
        url = "https://m.ctrip.com/restapi/soa2/12455/json/BnbCommentList?_fxpcqlniredt=09031121210974067355"
        html = HttpUtils.do_request("POST", url, headers, json.dumps(postData))
        data_json = json.loads(html.text)
        count = data_json['count']  # 评论总数量
        for d in data_json['clist']:
            dict_data = dict()
            dict_data['ctime'] = d['ctime']  # 时间
            dict_data['rate'] = d['rate']  # 评分
            dict_data['commentDesc'] = d['commentDesc']  # 评论标签
            dict_data['content'] = d['content']  # 评论内容
        pages = count // 10 + count % 10
        if pages % 10 != 0:
            pages = count // 10 + count % 10
        else:
            pages = count / 10
        for p in range(pageNum + 1, pages):
            self.get_comments(pid, p)
        # 评分
        dict_data['avgTotalRate'] = data_json['rateStat']['avgTotalRate']  # 总分
        dict_data['avgHygieneRate'] = data_json['rateStat']['avgHygieneRate']  # 房间卫生
        dict_data['avgLandlordServiceRate'] = data_json['rateStat']['avgLandlordServiceRate']  # 房东服务
        dict_data['avgFacilityRate'] = data_json['rateStat']['avgFacilityRate']  # 设施设备
        dict_data['avgEnvironmentRate'] = data_json['rateStat']['avgEnvironmentRate']  # 周边环境
        # 标签
        num = 0
        for d1 in data_json['bnbCommentTags']:
            num += 1
            dict_data['tagName_' + str(num)] = d1['tagName']  # 名称
            dict_data['tagName_' + str(num)] = d1['popularity']  # 数量


if __name__ == '__main__':
    wineshop = WineShop()
    for i in range(1, 10):
        wineshop.get_contents(i)